package fetcher

import (
	"golang.org/x/text/encoding"
	"bufio"
	"golang.org/x/net/html/charset"
	"net/http"
	"fmt"
	"golang.org/x/text/transform"
	"io/ioutil"
	"golang.org/x/text/encoding/unicode"
)




func determineEncoding(r *bufio.Reader) encoding.Encoding{
	bytes, err := r.Peek(1024)
	if err!=nil{

		return unicode.UTF8
	}

	e, _, _ := charset.DetermineEncoding(bytes, "")
	return  e
}


const UserAgent =  "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.87 Safari/537.36"


func Fetch(url string)([]byte,error){


	client := &http.Client{}
	//提交请求
	reqest, err := http.NewRequest("GET", url, nil)


	//增加header选项
	reqest.Header.Add("User-Agent", UserAgent)
	if err != nil{
		return nil,err
	}
	resp, e := client.Do(reqest)
	if e != nil{
		return nil,e
	}
	defer resp.Body.Close()



	if resp.StatusCode != http.StatusOK{
		fmt.Println("Error ：status code",resp.StatusCode)
		fmt.Println(resp.Body)
		return nil,fmt.Errorf("wrong status code: %d",resp.StatusCode)
	}

	bodyReader := bufio.NewReader(resp.Body)
	encoding := determineEncoding(bodyReader)
	uff8Reader := transform.NewReader(bodyReader, encoding.NewDecoder())
	return  ioutil.ReadAll(uff8Reader)
}